package com.bedpotato.thread;

import java.util.HashSet;
import java.util.Set;

import javax.annotation.Resource;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

import com.bedpotato.constants.ProxyClawerConstants;
import com.bedpotato.http.BasicHttpMethod;
import com.bedpotato.po.ProxyPO;
import com.bedpotato.service.ProxyClawerService;

@Component
public class SitediggerSiteClawerThread implements Runnable {
	Logger logger = Logger.getLogger(SitediggerSiteClawerThread.class);
	@Resource
	ProxyClawerService proxyClawerService;
	public static void main(String[] args) {
		SitediggerSiteClawerThread sitediggerSiteClawerThread = ProxyClawerConstants.applicationContext.getBean("sitediggerSiteClawerThread",SitediggerSiteClawerThread.class);
		sitediggerSiteClawerThread.run();
	}
	@Override
	public void run() {
		logger.info("Sitedigger信息抓取启动");
		Set<ProxyPO> proxySet = getSitediggerData();
		proxyClawerService.saveProxy(proxySet);
		logger.info("Sitedigger信息抓取结束");
	}

	public Set<ProxyPO> getSitediggerData() {
		BasicHttpMethod httpMethod = new BasicHttpMethod();
		Set<ProxyPO> proxySet = new HashSet<ProxyPO>();
		try {
			String pageStr = httpMethod
					.get("http://www.site-digger.com/html/articles/20110516/proxieslist.html");
			String keywordPageStr = StringUtils.substringBetween(pageStr,
					"<textarea", "</textarea>").trim();
			String[] contentStrArray = keywordPageStr.split("\n");
			for (String contentStr : contentStrArray) {
				if (!contentStr.isEmpty()
						&& StringUtils.isNumeric(contentStr.substring(0, 1))) {
					ProxyPO proxyPO = new ProxyPO();
					proxyPO.setIp(contentStr.substring(0,
							contentStr.indexOf(":")));
					proxyPO.setPort(contentStr.substring(
							contentStr.indexOf(":") + 1, contentStr.length()));
//					System.out.println(proxyPO.getIp() + ":"
//							+ proxyPO.getPort());
					proxySet.add(proxyPO);
				}
			}
		} catch (Exception e) {
			logger.error("Sitedigger "+e);
		}
		return proxySet;
	}
}
